R Notebook

This is an R Markdown Notebook. When you execute code within the notebook, the results appear beneath the code.

Try executing this chunk by clicking the Run button within the chunk or by placing your cursor inside it and pressing Ctrl+Shift+Enter. Add a new chunk by clicking the Insert Chunk button on the toolbar or by pressing Ctrl+Alt+I. When you save the notebook, an HTML file containing the code and output will be saved alongside it (click the Preview button or press Ctrl+Shift+K to preview the HTML file).

The preview shows you a rendered HTML copy of the contents of the editor. Consequently, unlike Knit, Preview does not run any R code chunks. Instead, the output of the chunk when it was last run in the editor is displayed.

========================================================

Course: CEBD1261 Big Data Infrstructure

By: Luciana Palucci

========================================================

# Define library packages to use and pre-install
# These packages must be installed at first if not already installed.
#install.packages("tidyverse", dependencies=TRUE)
#install.packages("plotly")
#install.packages("RCurl")
  
library(dplyr,verbose = getOption("verbose"))
library(tidyr)    # for reshaping data

package <U+393C><U+3E31>tidyr<U+393C><U+3E32> was built under R version 3.4.4
Attaching package: <U+393C><U+3E31>tidyr<U+393C><U+3E32>

The following object is masked from <U+393C><U+3E31>package:RCurl<U+393C><U+3E32>:

    complete

library(scales)     # for scale_y_continuous(label = percent)
library(ggthemes)   # for scale_fill_few('medium')

package <U+393C><U+3E31>ggthemes<U+393C><U+3E32> was built under R version 3.4.4

library(ggplot2)    # plotting data
library(ztable)     # format tables for reporting

Welcome to package ztable ver 0.1.8

library(tidyverse)

package <U+393C><U+3E31>tidyverse<U+393C><U+3E32> was built under R version 3.4.4-- Attaching packages --------------------------------------- tidyverse 1.2.1 --
v tibble  1.4.2     v purrr   0.2.4
v readr   1.1.1     v stringr 1.2.0
v tibble  1.4.2     v forcats 0.3.0
package <U+393C><U+3E31>tibble<U+393C><U+3E32> was built under R version 3.4.4package <U+393C><U+3E31>readr<U+393C><U+3E32> was built under R version 3.4.4package <U+393C><U+3E31>purrr<U+393C><U+3E32> was built under R version 3.4.4package <U+393C><U+3E31>forcats<U+393C><U+3E32> was built under R version 3.4.4-- Conflicts ------------------------------------------ tidyverse_conflicts() --
x readr::col_factor() masks scales::col_factor()
x tidyr::complete()   masks RCurl::complete()
x purrr::discard()    masks scales::discard()
x dplyr::filter()     masks stats::filter()
x purrr::invoke()     masks sparklyr::invoke()
x dplyr::lag()        masks stats::lag()

library(plotly)     # for interactive charts

package <U+393C><U+3E31>plotly<U+393C><U+3E32> was built under R version 3.4.4
Attaching package: <U+393C><U+3E31>plotly<U+393C><U+3E32>

The following object is masked from <U+393C><U+3E31>package:ggplot2<U+393C><U+3E32>:

    last_plot

The following object is masked from <U+393C><U+3E31>package:stats<U+393C><U+3E32>:

    filter

The following object is masked from <U+393C><U+3E31>package:graphics<U+393C><U+3E32>:

    layout

Read in Bureau of Transport Statistics dataset from

https://www.transtats.bts.gov/DL_SelectFields.asp?Table_ID=236&DB_Short_Name=On-Time

#Pipeline the data into Rstudio
#Get data from AWS to local copy of rstudio
#Source: https://aws.amazon.com/blogs/big-data/running-r-on-aws/
#library("RCurl") 
#data <- read.table(getURL("https://ec2-54-237-132-222.compute-1.amazonaws.com/Data/On_Time_Performance_2018.csv"), sep=",", #header=FALSE)
#head(data)

#read in the raw dataset
airports <- read.csv("C:/Users/Luciana/CEBD1261/CEBD1261_Project/Data/On_Time_Performance_2018.csv", header = TRUE)
# view summary for unusual data values
summary(airports$DepTime)

   Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
      1     922    1329    1334    1738    2400   16826 

Clean the dataset to exclude null values and time format that do not conform to standard, then add variable

to break time point into 15 min interval timepoint.

# Remove the rows by checking if not (!) an NA due to not meeting the date format
temp<- airports[!is.na(strptime(airports$DepTime, format = "%H%M")),]
class(temp$DepTime)

[1] "integer"

#Split time series data into time intervals by every 15 minutes and then plot the count
myDF<- mutate(temp, byGroup = as.matrix(cut(strptime(sub("(\\d+)(\\d{2})", "\\1:\\2", temp$DepTime), format = "%H:%M"), breaks="15 min") )) 
# unique(myDF$byGroup) #check unique values
# Extract time value from variable and make dataframes used for graphs and summary table
myDF["Time"] <- NA # This creates the new column filled with "NA"
myDF$Time <- strftime(as.POSIXlt(myDF$byGroup), format="%H:%M")

1. Big Data Example

===================================

Make dataframes used for graphs and summary table

# based on variable values
ds1 <- myDF[ which(myDF$DayofMonth == 1 & myDF$OriginState=="NY"), ]
ds1[with(ds1, order(Time)), ]

Plot the data using ggplot2 and pipes

# Prepare dataset and pipe the dataset into the graph
graph1 <- ds1 %>%
ggplot(aes(x = Time, y = Origin, group=OriginStateName, color = Carrier)) +
      geom_point() +  scale_x_discrete(breaks=c("00:00", "11:00", "12:00", "14:00","16:00","18:00", "20:00","23:00")  ) 
      labs(title = "New York State Airport Departures by Time Intervals",
           subtitle = "*The data frame is sent to the plot using pipes",
           y = "Airport Origin code",
           x = "Departure Time") + theme_bw(base_size = 8) +
      theme(axis.ticks = element_blank(), axis.text.x = element_blank())

NULL

#printout graph #1
graph1

Turn the gplot into an interactive chart.

ggplotly(graph1)

We recommend that you use the dev version of ggplot2 with `ggplotly()`
Install it with: `devtools::install_github('hadley/ggplot2')`

2. Sampling and BI Summary Example

===================================

Count Distribution by State and Origin City of Airport Departures by Carrier

# Create dataset with overall count distributions
ds2 <- myDF[ which(myDF$Carrier %in% c("9E","AA","AS","B6","DL","EV","F9")), ]
dfr_prop <- ds2 %>% 
  count(OriginState, Carrier) %>%  # group_by() & summarise(n=n()) are implicit
  mutate(prop = prop.table(n))     # prop = n/sum(n) works too /  prop.table(n)
as.data.frame(dfr_prop)            # print df

#Plot distribution percentage summary for month of January
gg_prop <- ggplot(data = data.frame()
                  , aes(x =OriginState, y = prop, fill = Carrier)) + 
  geom_bar(stat = 'identity', position = 'stack', alpha = 2/3) +  
  scale_y_continuous(labels = percent) +  
  scale_x_discrete(breaks=c("AK", "CA","FL", "IL", "GA","LA","MI","MO","NC","OH", "NJ","NY","PA","TX","WY")  ) +
  scale_fill_few('medium', drop = FALSE) +            # keep levels, if data is filtered
  labs(x = 'State', y = NULL, fill = 'Carrier',
       title = 'Distribution of Flights by State and Carrier for Month of January')  # title of graph
gg_prop %+%   dfr_prop  

End of program.